(ns ifeast.recipes.scrapers.epicurious
  "A scraper for the site http://epicurious.com which retrieves links to every single listed recipe."
  (:require [clojure.contrib.str-utils2 :as s])
  (:import java.net.URL
           java.util.zip.GZIPInputStream
           java.util.Scanner))

(def sitemap "http://www.epicurious.com/sitemap1.xml.gz")

(defn get-recipes []
  "Extract all recipe links from epicurious.com."
  (let [scanner (Scanner. (GZIPInputStream. (.openStream (URL. sitemap))))
        in (repeatedly #(.nextLine scanner))]
    (lazy-seq
      (map #(-> % (s/drop 6) (s/butlast 6)) 
           (filter #(.startsWith % "\t<loc>http://www.epicurious.com/recipes/food/views/") in)))))
